Project 2¶
Betsy Rosalen and Mikhail Groysman¶
Project Overview¶
- Identify a large 2-node network dataset—you can start with a dataset in a repository. Your data should meet the criteria that it consists of ties between and not within two (or more) distinct groups.
- Reduce the size of the network using a method such as the island method described in chapter 4 of social network analysis.
- What can you infer about each of the distinct groups?
You may work in a small group on the project.
Your code and analysis should be delivered in an IPython Notebook by end of day Monday 10/14. .
1. Choosing the data¶
We have decide to build on flight data from OpenFlights.org from our first project.
We already have Routes sets loaded. We will use Airplanes as our second node.
Loading the data¶
First import necessary packages for plotting graphs using NetworkX and Matplotlib and set up graph size parameters...
In [124]:
import networkx as nx
from networkx.algorithms import bipartite as bi
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams["figure.figsize"] = (15,12)
Read in Routes data to a Pandas dataframe.
In [125]:
routes_raw = pd.read_csv("Project1_Data/routes.dat",
header=None, sep=",",
usecols=[2, 4],
names=['Airline', 'Airport'])
#routes_raw1=routes_raw.drop_duplicates(subset=['Airline', 'Airport'],keep=False)
routes_raw=routes_raw.dropna()
routes_raw=routes_raw[routes_raw.Airport!="\\N"]
In [126]:
routes_raw.head()
Out[126]:
In [127]:
aa = routes_raw.copy()
In [128]:
#df['sum_A'] = df.groupby('B')['A'].transform('sum')
In [129]:
aa['weight'] = aa.groupby('Airline')['Airport'].transform('count')#.reset_index()
In [130]:
aa.head(10)
Out[130]:
In [131]:
#aa['Airline'] = 'Airline_' + aa.Airline.map(str)
#aa['Airport'] = 'Airport_' + aa.Airport.map(str)
In [132]:
aa.head(10)
Out[132]:
In [133]:
len(aa)
Out[133]:
In [134]:
airports_raw = pd.read_csv("Project1_Data/airports.dat",
header=None, sep=",",
usecols=[0,1, 2, 3, 4, 6, 7],
names=['Airport','Airport Name','City','Country','IATA', 'Latitude', 'Longitude'])
#airports_raw1=airports_raw[airports_raw.Country=='France']
In [135]:
airports_raw.head()
Out[135]:
In [136]:
len(airports_raw)
Out[136]:
In [137]:
airports = airports_raw.copy()
In [138]:
#airports['Airport']='Airport_' + airports.IATA.map(str)
airports['Airport']=airports.IATA
In [139]:
aa_merged = pd.merge(aa,airports,left_on='Airport',right_on='Airport',how='inner')
In [140]:
aa_merged.head(20)
Out[140]:
In [141]:
airport_nodes = aa_merged[['Airport','Airport Name','City','Country','IATA','Latitude','Longitude']].drop_duplicates()
In [142]:
airport_nodes.head()
Out[142]:
Create NetworkX Graph Object¶
...
In [143]:
B = nx.Graph()
B.add_nodes_from(aa_merged['Airline'], bipartite=0)
B.add_nodes_from(aa_merged['Airport'], bipartite=1)
B.add_weighted_edges_from([tuple(d) for d in aa_merged[['Airline','Airport','weight']].values])
nx.set_node_attributes(B, airport_nodes.set_index('Airport').to_dict('index'))
Let's double check if our attributes were saved in the graph data.
In [144]:
list(B.nodes(data=True))[:3]
Out[144]:
In [145]:
list(B.nodes(data=True))[-2:]
Out[145]:
In [146]:
B.get_edge_data('Airline_AER','Airport_KZN')
In [147]:
print(nx.info(B))
In [148]:
airline_nodes = {n for n, d in B.nodes(data=True) if d['bipartite']==0}
airport_nodes = set(B) - airline_nodes
In [149]:
nx.is_connected(B)
Out[149]:
In [150]:
bi.is_bipartite(B)
Out[150]:
In [151]:
print(bi.density(B, airline_nodes))
In [152]:
print(bi.density(B, airport_nodes))
Island Method on Graph with Both Node Types¶
In [153]:
'''
# textbook function doesn't work...
def trim_edges(g, weight=1):
g2=nx.Graph()
for f, to, edata in g.edges(data=True):
if edata['weight'] > weight:
g2.add_edge(f, to, edata)
return g2
'''
def trim_edges(g, weight=1):
g2=nx.Graph()
my_list=[]
my_list1=[]
for f, to, edata in g.edges(data=True):
if edata['weight'] > weight:
my_list.append(f)
my_list1.append(to)
g2.add_edge(f,to,attr_dict={weight:edata['weight']})
g2.add_nodes_from(my_list, bipartite=0)
g2.add_nodes_from(my_list1, bipartite=1)
return g2
def island_method(g, iterations=5):
weights = [edata['weight'] for f, to, edata in g.edges(data=True)]
mn=int(min(weights))
mx=int(max(weights))
#compute the size of the step, so we get a reasonable step in iterations
step=int((mx-mn)/iterations)
return [[threshold, trim_edges(g, threshold)] for threshold in range(mn,mx,step)]
In [154]:
island = island_method(B)
for i in island:
# print the threshold level, size of the graph, and number of connected components
print(i[0], len(i[1]), len(list(nx.connected_component_subgraphs(i[1]))))
Largest Subgraph¶
In [186]:
G0=max(nx.connected_component_subgraphs(island[0][1]), key=len)
In [187]:
colors = []
for node, data in G0.nodes(data=True):
if data['bipartite'] == 1:
colors.append('cornflowerblue') # Airports in Blue
else:
colors.append('orchid') # Airlines in pink
#nx.draw(G, with_labels=True, node_size=100, node_color=colors, font_size=10, font_family="calibri", width=1,
# edge_color="black", alpha=0.8)
nx.draw(G0,
with_labels=True,
node_color=colors,
node_size=200,
font_size=10,
font_weight='bold',
edge_color="skyblue",
alpha=0.5)
Second Largest Subgraph¶
In [169]:
G1=island[1][1]
In [170]:
colors = []
for node, data in G1.nodes(data=True):
if data['bipartite'] == 1:
colors.append('cornflowerblue') # Airports in Blue
else:
colors.append('orchid') # Airlines in pink
#nx.draw(G, with_labels=True, node_size=100, node_color=colors, font_size=10, font_family="calibri", width=1,
# edge_color="black", alpha=0.8)
nx.draw(G1,
with_labels=True,
node_color=colors,
node_size=200,
font_size=10,
font_weight='bold',
edge_color="skyblue",
alpha=0.5)
Third Largest Subgraph¶
In [157]:
G2=island[2][1]
In [158]:
colors = []
for node, data in G2.nodes(data=True):
if data['bipartite'] == 1:
colors.append('cornflowerblue') # Airports in Blue
else:
colors.append('orchid') # Airlines in pink
#nx.draw(G, with_labels=True, node_size=100, node_color=colors, font_size=10, font_family="calibri", width=1,
# edge_color="black", alpha=0.8)
nx.draw(G2,
with_labels=True,
node_color=colors,
node_size=200,
font_size=10,
font_weight='bold',
edge_color="skyblue",
alpha=0.5)
In [159]:
G3=island[3][1]
In [160]:
colors = []
for node, data in G3.nodes(data=True):
if data['bipartite'] == 1:
colors.append('cornflowerblue') # Airports in Blue
else:
colors.append('orchid') # Airlines in pink
#nx.draw(G, with_labels=True, node_size=100, node_color=colors, font_size=10, font_family="calibri", width=1,
# edge_color="black", alpha=0.8)
nx.draw(G3,
with_labels=True,
node_color=colors,
node_size=200,
font_size=10,
font_weight='bold',
edge_color="skyblue",
alpha=0.5)
Airport Nodes Only¶
In [161]:
# compute an affiliation network of the Airports
airports = bi.weighted_projected_graph(B, airport_nodes)
# Find the largest connected subgraph in the network
#users_subgraph = nx.connected_component_subgraphs(users)[0] # textbook code doesn't work
airports_subgraph = max(nx.connected_component_subgraphs(airports), key=len) # alternate method
airports_subgraph.name = "Airports"
print(nx.info(airports_subgraph))
In [162]:
nx.is_connected(airports_subgraph)
Out[162]:
In [163]:
bi.is_bipartite(airports_subgraph)
Out[163]:
In [164]:
airport_islands=island_method(airports_subgraph)
for i in airport_islands:
# print the threshold level, size of the graph, and number of connected components
print(i[0], len(i[1]), len(list(nx.connected_component_subgraphs(i[1]))))
In [172]:
API=max(nx.connected_component_subgraphs(airport_islands[0][1]), key=len)
In [182]:
# set plot size
plt.rcParams["figure.figsize"] = (15,15)
nx.draw(API,
with_labels=True,
node_color='cornflowerblue',
node_size=200,
font_size=10,
font_weight='bold',
edge_color="cornflowerblue",
alpha=0.5)
Airline Nodes Only¶
In [175]:
# compute an affiliation network of the Airlines
airlines = bi.weighted_projected_graph(B, airline_nodes)
# Find the largest connected subgraph in the network
#users_subgraph = nx.connected_component_subgraphs(users)[0] # textbook code doesn't work
airlines_subgraph = max(nx.connected_component_subgraphs(airlines), key=len) # alternate method
airlines_subgraph.name = "Airlines"
print(nx.info(airlines_subgraph))
In [176]:
nx.is_connected(airlines_subgraph)
Out[176]:
In [177]:
bi.is_bipartite(airlines_subgraph)
Out[177]:
In [178]:
airline_islands=island_method(airlines_subgraph)
for i in airline_islands:
# print the threshold level, size of the graph, and number of connected components
print(i[0], len(i[1]), len(list(nx.connected_component_subgraphs(i[1]))))
In [179]:
ALI=max(nx.connected_component_subgraphs(airline_islands[0][1]), key=len)
In [183]:
# set plot size
plt.rcParams["figure.figsize"] = (15,15)
nx.draw(ALI,
with_labels=True,
node_color='orchid',
node_size=200,
font_size=10,
font_weight='bold',
edge_color="orchid",
alpha=0.5)
